#！D:\MyWork\Program\python
# -*- coding: utf-8 -*-
#更新古籍内容的格式
import requests_html
import re
from bs4 import BeautifulSoup
import random
import time
import traceback
import pymysql
import gc
import os


def updateBookText():
    sql="select book_id,`index` from books where 目录 is not null  and isupdate=0 order by id "
    #sql="select book_id,`index` from books where book_id='5b39d81dba95566b43018fd6'  and isupdate=0 order by id desc  "
    cur.execute(sql)
    books=cur.fetchall()
    print(len(books))
    user_agent=requests_html.user_agent()


    values=[]
    count=0
    for book in books:
        cur_book_id=book[0]
        index=book[1]
        print("book_id:"+cur_book_id+",index:"+str(index))

        try:
            session=requests_html.HTMLSession()
            headers={"User-Agent":user_agent,
                     "Cookie":"sid=c33acb969ed91b06e2a98e55d405c4e8; sys_lang=zh; UM_distinctid=172f369309e477-0121f77c4e3b6-551d3911-144000-172f369309f4b2; _cbshow=true; CNZZDATA1277615859=1385762309-1593221660-%7C1593248835; aha=WEBmsg8618634976469LOGIN; bearer=eyJhbGciOiJIUzUxMiJ9.eyJzdWIiOiJBSDEzNDI1MjY4MDczNTgiLCJjcmVhdGVkIjoxNTkzMjQ5ODQ2ODEzLCJleHAiOjE1OTU4NDE4NDYsInVzZXJpZCI6MTM0MjUyNn0.BTNaUgUmhnMuPuSNBemw3m5vJsYfnHiGluQUnpnJ51-TcFUip1EbKigeb2nIZge4AJSQwzwy1GSyR5iChoV98A; free=free; page.prev.id=11v96nj13kbxfluzc; page.id=11v96nj1skbxftmmm; refer.url=https%3A%2F%2Fwww.allhistory.com%2Fbook%2Fhome-list%2F; connect.sid=s%3A3HhrLMtGhtlwHp0OTICuZTiO1UqGwWS9.nJ%2B%2B1tpfya8vb9Yntezcy77jE5YEJk6HAg6bRZkinSE"
                     }
            text_url='https://www.allhistory.com/book/read?id='+cur_book_id+'&restrict=text'
            text=session.get(text_url,headers=headers)
            text.html.render(sleep=2,timeout=15)
            text=text.html.html
            #print(text)
            time.sleep(1)

            catalogs=r'<li class="book-menu-list-item.*?title="(.*?)".*?>';catalogs=re.compile(catalogs,re.S);catalogs=catalogs.findall(text)
            vids=r'<li class="book-menu-list-item.*?data-volumnid="(.*?)".*?>';vids=re.compile(vids,re.S);vids=vids.findall(text)
            #print(len(vids))
            if len(vids)==0:
                print("未获取到详情页id")
                session.close()
                time.sleep(1)
                continue
            vid_url='https://www.allhistory.com/book/read?id='+cur_book_id+'&vid='+vids[index]+'&restrict=text&page=1&mode=undefined&single=ORIGINAL&left=ZH_CC&right='
            # session=requests_html.HTMLSession()
            # headers={"User-Agent":user_agent,
            #          "Cookie":"sid=c33acb969ed91b06e2a98e55d405c4e8; sys_lang=zh; UM_distinctid=172f369309e477-0121f77c4e3b6-551d3911-144000-172f369309f4b2; _cbshow=true; CNZZDATA1277615859=1385762309-1593221660-%7C1593248835; aha=WEBmsg8618634976469LOGIN; bearer=eyJhbGciOiJIUzUxMiJ9.eyJzdWIiOiJBSDEzNDI1MjY4MDczNTgiLCJjcmVhdGVkIjoxNTkzMjQ5ODQ2ODEzLCJleHAiOjE1OTU4NDE4NDYsInVzZXJpZCI6MTM0MjUyNn0.BTNaUgUmhnMuPuSNBemw3m5vJsYfnHiGluQUnpnJ51-TcFUip1EbKigeb2nIZge4AJSQwzwy1GSyR5iChoV98A; free=free; page.prev.id=11v96nj13kbxfluzc; page.id=11v96nj1skbxftmmm; refer.url=https%3A%2F%2Fwww.allhistory.com%2Fbook%2Fhome-list%2F; connect.sid=s%3A3HhrLMtGhtlwHp0OTICuZTiO1UqGwWS9.nJ%2B%2B1tpfya8vb9Yntezcy77jE5YEJk6HAg6bRZkinSE"
            #          }
            data=session.get(vid_url,headers=headers)
            data.html.render(sleep=2,timeout=10)
            data=data.html.html.replace('<br><br>','\n').replace('<br>','\n')
            #print(data)
            soup = BeautifulSoup(data,'lxml')
            datas=[]
            #将每个详情页的内容放入集合
            for x in soup.find_all('div',class_="dls-p"):
                if x.find('p',class_='p1'):
                    a=x.find('p',class_='p1').get_text()
                else:
                    a=x.get_text()
                #a为标签内容
                datas.append(a)
                del a
            #每个详情页内容拼接
            #print(datas)
            if len(datas)==0:
                print("内容为空")
                session.close()
                time.sleep(2)
                continue
            datas='\n'.join(datas)
            #print(datas)
            value=(datas,vids[index],1,cur_book_id,index)
            values.append(value)

            update = "update books set `内容`=%s,vid=%s,isupdate=%s where book_id=%s and `index`=%s"
            cur.execute(update,value)
            conn.commit()
            #print("提交修改")
            session.close()
            time.sleep(random.randint(2,4))
        except:
            print("异常book_id:"+cur_book_id+",index:"+str(index))
            session.close()
            #traceback.print_exc()
            continue
    #update = "update books set `内容`=%s where book_id=%s and `index`=%s"











if __name__ == '__main__':
    try:
        conn = pymysql.Connect(host = '10.170.128.56',
                               port = 3306,
                               user = 'root',
                               passwd = '123456',
                               db = 'books',
                               charset='utf8mb4')
        cur = conn.cursor()


        updateBookText()

    finally:
        cur.close()
        conn.close()